<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=9"/>
<title>Reranker Framework (ReFr): hadooputil.py Source File</title>

<link href="tabs.css" rel="stylesheet" type="text/css"/>
<link href="doxygen.css" rel="stylesheet" type="text/css" />

<link href="search/search.css" rel="stylesheet" type="text/css"/>
<script type="text/javascript" src="jquery.js"></script>
<script type="text/javascript" src="search/search.js"></script>
<script type="text/javascript">
  $(document).ready(function() { searchBox.OnSelectItem(0); });
</script>

</head>
<body>
<div id="top"><!-- do not remove this div! -->


<div id="titlearea">
<table cellspacing="0" cellpadding="0">
 <tbody>
 <tr style="height: 56px;">
  
  
  <td style="padding-left: 0.5em;">
   <div id="projectname">Reranker Framework (ReFr)
   
   </div>
   <div id="projectbrief">Reranking framework for structure prediction and discriminative language modeling</div>
  </td>
  
  
  
 </tr>
 </tbody>
</table>
</div>

<!-- Generated by Doxygen 1.7.6.1 -->
<script type="text/javascript">
var searchBox = new SearchBox("searchBox", "search",false,'Search');
</script>
  <div id="navrow1" class="tabs">
    <ul class="tablist">
      <li><a href="index.html"><span>Main&#160;Page</span></a></li>
      <li><a href="namespaces.html"><span>Namespaces</span></a></li>
      <li><a href="annotated.html"><span>Classes</span></a></li>
      <li class="current"><a href="files.html"><span>Files</span></a></li>
      <li>
        <div id="MSearchBox" class="MSearchBoxInactive">
        <span class="left">
          <img id="MSearchSelect" src="search/mag_sel.png"
               onmouseover="return searchBox.OnSearchSelectShow()"
               onmouseout="return searchBox.OnSearchSelectHide()"
               alt=""/>
          <input type="text" id="MSearchField" value="Search" accesskey="S"
               onfocus="searchBox.OnSearchFieldFocus(true)" 
               onblur="searchBox.OnSearchFieldFocus(false)" 
               onkeyup="searchBox.OnSearchFieldChange(event)"/>
          </span><span class="right">
            <a id="MSearchClose" href="javascript:searchBox.CloseResultsWindow()"><img id="MSearchCloseImg" border="0" src="search/close.png" alt=""/></a>
          </span>
        </div>
      </li>
    </ul>
  </div>
  <div id="navrow2" class="tabs2">
    <ul class="tablist">
      <li><a href="files.html"><span>File&#160;List</span></a></li>
      <li><a href="globals.html"><span>File&#160;Members</span></a></li>
    </ul>
  </div>
</div>
<div class="header">
  <div class="headertitle">
<div class="title">hadooputil.py</div>  </div>
</div><!--header-->
<div class="contents">
<a href="hadooputil_8py.html">Go to the documentation of this file.</a><div class="fragment"><pre class="fragment"><a name="l00001"></a><a class="code" href="namespacehadooputil.html">00001</a> <span class="comment">#!/usr/bin/env python</span>
<a name="l00002"></a>00002 <span class="comment">#-----------------------------------------------------------------------</span>
<a name="l00003"></a>00003 <span class="comment"># Copyright 2012, Google Inc.</span>
<a name="l00004"></a>00004 <span class="comment"># All rights reserved.</span>
<a name="l00005"></a>00005 <span class="comment"># </span>
<a name="l00006"></a>00006 <span class="comment"># Redistribution and use in source and binary forms, with or without</span>
<a name="l00007"></a>00007 <span class="comment"># modification, are permitted provided that the following conditions are</span>
<a name="l00008"></a>00008 <span class="comment"># met:</span>
<a name="l00009"></a>00009 <span class="comment"># </span>
<a name="l00010"></a>00010 <span class="comment">#   * Redistributions of source code must retain the above copyright</span>
<a name="l00011"></a>00011 <span class="comment">#     notice, this list of conditions and the following disclaimer.</span>
<a name="l00012"></a>00012 <span class="comment">#   * Redistributions in binary form must reproduce the above</span>
<a name="l00013"></a>00013 <span class="comment">#     copyright notice, this list of conditions and the following disclaimer</span>
<a name="l00014"></a>00014 <span class="comment">#     in the documentation and/or other materials provided with the</span>
<a name="l00015"></a>00015 <span class="comment">#     distribution.</span>
<a name="l00016"></a>00016 <span class="comment">#   * Neither the name of Google Inc. nor the names of its</span>
<a name="l00017"></a>00017 <span class="comment">#     contributors may be used to endorse or promote products derived from</span>
<a name="l00018"></a>00018 <span class="comment">#     this software without specific prior written permission.</span>
<a name="l00019"></a>00019 <span class="comment">#</span>
<a name="l00020"></a>00020 <span class="comment"># THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</span>
<a name="l00021"></a>00021 <span class="comment"># &quot;AS IS&quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</span>
<a name="l00022"></a>00022 <span class="comment"># LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</span>
<a name="l00023"></a>00023 <span class="comment"># A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</span>
<a name="l00024"></a>00024 <span class="comment"># OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</span>
<a name="l00025"></a>00025 <span class="comment"># SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</span>
<a name="l00026"></a>00026 <span class="comment"># LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,           </span>
<a name="l00027"></a>00027 <span class="comment"># DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY           </span>
<a name="l00028"></a>00028 <span class="comment"># THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</span>
<a name="l00029"></a>00029 <span class="comment"># (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</span>
<a name="l00030"></a>00030 <span class="comment"># OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</span>
<a name="l00031"></a>00031 <span class="comment"># -----------------------------------------------------------------------------</span>
<a name="l00032"></a>00032 <span class="comment">## @file hadooputil.py</span>
<a name="l00033"></a>00033 <span class="comment">#  A set of utilities to help interface with Hadoop.</span>
<a name="l00034"></a>00034 <span class="comment">#</span>
<a name="l00035"></a>00035 
<a name="l00036"></a>00036 <span class="keyword">import</span> os,sys, math
<a name="l00037"></a>00037 <span class="keyword">import</span> pyutil
<a name="l00038"></a>00038 
<a name="l00039"></a>00039 <span class="comment">## @class HadoopInterface</span>
<a name="l00040"></a>00040 <span class="comment">#  A simple class interface for running hadoop commands.</span>
<a name="l00041"></a>00041 <span class="comment">#</span>
<a name="l00042"></a><a class="code" href="classhadooputil_1_1_hadoop_interface.html">00042</a> <span class="keyword">class </span><a class="code" href="classhadooputil_1_1_hadoop_interface.html" title="A simple class interface for running hadoop commands.">HadoopInterface</a>:
<a name="l00043"></a><a class="code" href="classhadooputil_1_1_hadoop_interface.html#a7672c1241563bfeeefe9fcab238ce898">00043</a>   <span class="keyword">def </span><a class="code" href="classhadooputil_1_1_hadoop_interface.html#a52a0a58a92e6a475a0187621e0a80c7e">__init__</a>(self, hbasedir, streamingloc, minsplitsize, tasktimeout, libpath):
<a name="l00044"></a>00044     self.<a class="code" href="classhadooputil_1_1_hadoop_interface.html#a1cf9db2b607bd7159d0abd20e16f98e8">hadoopmr_</a> = hbasedir + <span class="stringliteral">&quot;/bin/hadoop &quot;</span>
<a name="l00045"></a>00045     self.<a class="code" href="classhadooputil_1_1_hadoop_interface.html#a1cf9db2b607bd7159d0abd20e16f98e8">hadoopmr_</a> += <span class="stringliteral">&quot; jar &quot;</span> + streamingloc
<a name="l00046"></a>00046     <span class="keywordflow">if</span> (int(minsplitsize) &gt; 0):
<a name="l00047"></a>00047       self.<a class="code" href="classhadooputil_1_1_hadoop_interface.html#a1cf9db2b607bd7159d0abd20e16f98e8">hadoopmr_</a> += <span class="stringliteral">&quot; -Dmapred.min.split.size=&quot;</span> + str(minsplitsize)
<a name="l00048"></a>00048     <span class="keywordflow">if</span> (int(tasktimeout) &gt;= 0):
<a name="l00049"></a>00049       self.<a class="code" href="classhadooputil_1_1_hadoop_interface.html#a1cf9db2b607bd7159d0abd20e16f98e8">hadoopmr_</a> += <span class="stringliteral">&quot; -Dmapred.task.timeout=&quot;</span> + str(tasktimeout)
<a name="l00050"></a>00050     self.<a class="code" href="classhadooputil_1_1_hadoop_interface.html#a89be5c343162d881a0fea7983dedc181">hadooplibpath_</a> = <span class="stringliteral">&quot;&quot;</span>
<a name="l00051"></a>00051     <span class="keywordflow">if</span> (libpath):
<a name="l00052"></a>00052       self.<a class="code" href="classhadooputil_1_1_hadoop_interface.html#a89be5c343162d881a0fea7983dedc181">hadooplibpath_</a> = <span class="stringliteral">&quot; --cmdenv LD_LIBRARY_PATH=&quot;</span> + libpath + <span class="stringliteral">&quot; &quot;</span>
<a name="l00053"></a>00053     self.<a class="code" href="classhadooputil_1_1_hadoop_interface.html#a9bed5f81e0d212ee5aee3a5773b322c9">hadoopfs_</a> = hbasedir + <span class="stringliteral">&quot;/bin/hadoop fs &quot;</span>
<a name="l00054"></a>00054     self.<a class="code" href="classhadooputil_1_1_hadoop_interface.html#a7672c1241563bfeeefe9fcab238ce898">hadooptest_</a> = hbasedir + <span class="stringliteral">&quot;/bin/hadoop fs -test &quot;</span>
<a name="l00055"></a>00055     self.<a class="code" href="classhadooputil_1_1_hadoop_interface.html#af9dab589bb0d3352bf79c662b05bdc7a">hadoopcat_</a> = hbasedir + <span class="stringliteral">&quot;/bin/hadoop fs -cat &quot;</span> 
<a name="l00056"></a>00056     self.<a class="code" href="classhadooputil_1_1_hadoop_interface.html#a873f248f481b62bcfde3e2d65a6e7687">hadoopput_</a> = hbasedir + <span class="stringliteral">&quot;/bin/hadoop fs -put &quot;</span>
<a name="l00057"></a>00057     self.<a class="code" href="classhadooputil_1_1_hadoop_interface.html#a5754d6ed7d43209ee9cff402bb79cfbf">hadoopmove_</a> = hbasedir + <span class="stringliteral">&quot;/bin/hadoop fs -moveFromLocal &quot;</span>
<a name="l00058"></a>00058     self.<a class="code" href="classhadooputil_1_1_hadoop_interface.html#aaf2550ac3f9b2773632470cfb3bc47ff">hadoopget_</a> = hbasedir + <span class="stringliteral">&quot;/bin/hadoop fs -get &quot;</span>
<a name="l00059"></a>00059     self.<a class="code" href="classhadooputil_1_1_hadoop_interface.html#a1d5ee8df6509159441eb585309893e23">hadoopmkdir_</a> = hbasedir + <span class="stringliteral">&quot;/bin/hadoop fs -mkdir &quot;</span>
<a name="l00060"></a>00060     self.<a class="code" href="classhadooputil_1_1_hadoop_interface.html#abc211f355e6f085d13e0cf0379b2836f">hadooprmr_</a> = hbasedir + <span class="stringliteral">&quot;/bin/hadoop fs -rmr &quot;</span>
<a name="l00061"></a>00061 
<a name="l00062"></a>00062   <span class="comment">## Check if a directory exists on HDFS.</span>
<a name="l00063"></a>00063   <span class="comment">#  @param[in] directory name of directory to check</span>
<a name="l00064"></a>00064   <span class="comment">#  @return True if directory exits.</span>
<a name="l00065"></a><a class="code" href="classhadooputil_1_1_hadoop_interface.html#a28d45b5f55d5140ccb59490502bca4d2">00065</a>   <span class="keyword">def </span><a class="code" href="classhadooputil_1_1_hadoop_interface.html#a28d45b5f55d5140ccb59490502bca4d2" title="Check if a directory exists on HDFS.">CheckHDir</a>(self, directory):
<a name="l00066"></a>00066     test_str = self.<a class="code" href="classhadooputil_1_1_hadoop_interface.html#a7672c1241563bfeeefe9fcab238ce898">hadooptest_</a> + <span class="stringliteral">&quot;-d &quot;</span> + directory
<a name="l00067"></a>00067     <span class="keywordflow">return</span> (pyutil.runCommand(test_str) == 0)
<a name="l00068"></a>00068 
<a name="l00069"></a>00069   <span class="comment">## Function to check for the existence of a directory on the HDFS.</span>
<a name="l00070"></a>00070   <span class="comment">#  @param[in] directory Direcotry to check.</span>
<a name="l00071"></a>00071   <span class="comment">#  @param[in] remove Remove the directory if it exists.</span>
<a name="l00072"></a>00072   <span class="comment">#  @return True if it did not exist or was removed.</span>
<a name="l00073"></a><a class="code" href="classhadooputil_1_1_hadoop_interface.html#ad635e5359e19312ffe2f65b9b106be4f">00073</a>   <span class="keyword">def </span><a class="code" href="classhadooputil_1_1_hadoop_interface.html#ad635e5359e19312ffe2f65b9b106be4f" title="Function to check for the existence of a directory on the HDFS.">CheckRemoveHDir</a>(self, directory, remove):
<a name="l00074"></a>00074     nodir = <span class="keyword">False</span>
<a name="l00075"></a>00075     <span class="keywordflow">if</span> (self.<a class="code" href="classhadooputil_1_1_hadoop_interface.html#a28d45b5f55d5140ccb59490502bca4d2" title="Check if a directory exists on HDFS.">CheckHDir</a>(directory)):
<a name="l00076"></a>00076       <span class="keywordflow">if</span> (remove):
<a name="l00077"></a>00077         rm_str = self.<a class="code" href="classhadooputil_1_1_hadoop_interface.html#abc211f355e6f085d13e0cf0379b2836f">hadooprmr_</a> + directory
<a name="l00078"></a>00078         pyutil.runCommand(rm_str)
<a name="l00079"></a>00079         nodir = <span class="keyword">True</span>
<a name="l00080"></a>00080     <span class="keywordflow">else</span>:
<a name="l00081"></a>00081       nodir = <span class="keyword">True</span>
<a name="l00082"></a>00082     <span class="keywordflow">return</span> nodir
<a name="l00083"></a>00083 
<a name="l00084"></a>00084   <span class="comment">## Function to check for a file on HDFS.</span>
<a name="l00085"></a>00085   <span class="comment">#  @param[in] filename The file to check for.</span>
<a name="l00086"></a><a class="code" href="classhadooputil_1_1_hadoop_interface.html#a6c2c6c040fa0a2fc49dc595ea2f953ad">00086</a>   <span class="keyword">def </span><a class="code" href="classhadooputil_1_1_hadoop_interface.html#a6c2c6c040fa0a2fc49dc595ea2f953ad" title="Function to check for a file on HDFS.">CheckHDFSFile</a>(self, filename):
<a name="l00087"></a>00087     hdinput_test = self.<a class="code" href="classhadooputil_1_1_hadoop_interface.html#a7672c1241563bfeeefe9fcab238ce898">hadooptest_</a> + <span class="stringliteral">&quot;-e &quot;</span> + filename
<a name="l00088"></a>00088     <span class="keywordflow">return</span> (pyutil.runCommand(hdinput_test) == 0)
<a name="l00089"></a>00089 
<a name="l00090"></a>00090   <span class="comment">## Check for an input file and prepare it for MR processing.</span>
<a name="l00091"></a>00091   <span class="comment">#  @param[in] inputfile Name of the local file to prepare.</span>
<a name="l00092"></a>00092   <span class="comment">#  @param[in] hdfsinputdir HDFS directory for data staging.</span>
<a name="l00093"></a>00093   <span class="comment">#  @param[in] outputdir Local file system directory for the output.</span>
<a name="l00094"></a>00094   <span class="comment">#  @param[in] force Reprocess data even if files already exist.</span>
<a name="l00095"></a>00095   <span class="comment">#  @param[in] uncompress Uncompress data, if compressed, before running.</span>
<a name="l00096"></a>00096   <span class="comment">#</span>
<a name="l00097"></a>00097   <span class="comment">#  @return A MR input string.</span>
<a name="l00098"></a>00098   <span class="comment">#  Stage the input data for MapReduce processing.</span>
<a name="l00099"></a>00099   <span class="comment">#  If we are uncompressing compressed files, then we move uncompressed data</span>
<a name="l00100"></a>00100   <span class="comment">#  HDFS; otherwise, we simply copy the data to HDFS.</span>
<a name="l00101"></a>00101   <span class="comment">#</span>
<a name="l00102"></a><a class="code" href="classhadooputil_1_1_hadoop_interface.html#ad1d4eb787ed9f637d0de21df4f0ff15c">00102</a>   <span class="keyword">def </span><a class="code" href="classhadooputil_1_1_hadoop_interface.html#ad1d4eb787ed9f637d0de21df4f0ff15c" title="Check for an input file and prepare it for MR processing.">CheckInputFile</a>(self, inputfile, hdfsinputdir, outputdir, force, uncompress):
<a name="l00103"></a>00103     input_file_list = <span class="stringliteral">&quot;&quot;</span>
<a name="l00104"></a>00104     <span class="keywordflow">if</span> (inputfile.endswith(<span class="stringliteral">&quot;.gz&quot;</span>) <span class="keywordflow">and</span> uncompress):
<a name="l00105"></a>00105       input_filename = os.path.basename(inputfile).replace(<span class="stringliteral">&quot;.gz&quot;</span>,<span class="stringliteral">&quot;&quot;</span>)
<a name="l00106"></a>00106     <span class="keywordflow">else</span>:
<a name="l00107"></a>00107       input_filename = os.path.basename(inputfile)
<a name="l00108"></a>00108     pyutil.printDebug(1, <span class="stringliteral">&quot;Processing input &quot;</span> + input_filename + <span class="stringliteral">&quot;\n&quot;</span>)
<a name="l00109"></a>00109     <span class="comment"># Copy the input data to HDFS</span>
<a name="l00110"></a>00110     <span class="comment"># Check that the input data exists and move to HDFS if necessary.</span>
<a name="l00111"></a>00111     hdfsinputfile = hdfsinputdir + <span class="stringliteral">&quot;/&quot;</span> + input_filename
<a name="l00112"></a>00112     <span class="keywordflow">if</span> (<span class="keywordflow">not</span> self.<a class="code" href="classhadooputil_1_1_hadoop_interface.html#a6c2c6c040fa0a2fc49dc595ea2f953ad" title="Function to check for a file on HDFS.">CheckHDFSFile</a>(hdfsinputfile) <span class="keywordflow">and</span> <span class="keywordflow">not</span> force):
<a name="l00113"></a>00113       <span class="keywordflow">if</span> (<span class="keywordflow">not</span> self.<a class="code" href="classhadooputil_1_1_hadoop_interface.html#a28d45b5f55d5140ccb59490502bca4d2" title="Check if a directory exists on HDFS.">CheckHDir</a>(hdfsinputdir)):
<a name="l00114"></a>00114         pyutil.runCommand(self.<a class="code" href="classhadooputil_1_1_hadoop_interface.html#a1d5ee8df6509159441eb585309893e23">hadoopmkdir_</a> + hdfsinputdir)
<a name="l00115"></a>00115       <span class="keywordflow">if</span> (inputfile.endswith(<span class="stringliteral">&quot;.gz&quot;</span>) <span class="keywordflow">and</span> uncompress):
<a name="l00116"></a>00116         new_input = outputdir + <span class="stringliteral">&quot;/&quot;</span> + input_filename
<a name="l00117"></a>00117         unzipcmd = <span class="stringliteral">&quot;gunzip -c &quot;</span> + inputfile + <span class="stringliteral">&quot; &gt; &quot;</span> + new_input
<a name="l00118"></a>00118         <span class="keywordflow">if</span> (pyutil.runCommand(unzipcmd) != 0 <span class="keywordflow">or</span> force):
<a name="l00119"></a>00119           pyutil.printError(12, <span class="stringliteral">&quot;Unable to unzip file: &quot;</span> + inputfile)
<a name="l00120"></a>00120         pyutil.runCommand(self.<a class="code" href="classhadooputil_1_1_hadoop_interface.html#a5754d6ed7d43209ee9cff402bb79cfbf">hadoopmove_</a> + new_input + <span class="stringliteral">&quot; &quot;</span> + hdfsinputdir)
<a name="l00121"></a>00121         input_file_list += <span class="stringliteral">&quot; --input &quot;</span> + hdfsinputdir + <span class="stringliteral">&quot;/&quot;</span> + input_filename
<a name="l00122"></a>00122       <span class="keywordflow">else</span>:
<a name="l00123"></a>00123         pyutil.runCommand(self.<a class="code" href="classhadooputil_1_1_hadoop_interface.html#a873f248f481b62bcfde3e2d65a6e7687">hadoopput_</a> + inputfile + <span class="stringliteral">&quot; &quot;</span> + hdfsinputdir)
<a name="l00124"></a>00124         input_file_list += <span class="stringliteral">&quot; --input &quot;</span> + hdfsinputdir + <span class="stringliteral">&quot;/&quot;</span> + input_filename
<a name="l00125"></a>00125       <span class="keywordflow">if</span> (<span class="keywordflow">not</span> self.<a class="code" href="classhadooputil_1_1_hadoop_interface.html#a6c2c6c040fa0a2fc49dc595ea2f953ad" title="Function to check for a file on HDFS.">CheckHDFSFile</a>(hdfsinputfile)):
<a name="l00126"></a>00126         pyutil.printError(10, <span class="stringliteral">&quot;Unable to create input on HDFS: &quot;</span> + hdfsinputfile)
<a name="l00127"></a>00127     <span class="keywordflow">else</span>:
<a name="l00128"></a>00128       input_file_list += <span class="stringliteral">&quot; --input &quot;</span> + hdfsinputdir + <span class="stringliteral">&quot;/&quot;</span> + input_filename
<a name="l00129"></a>00129       pyutil.printDebug(5, <span class="stringliteral">&quot;Found file on HDFS: &quot;</span> + hdfsinputdir + <span class="stringliteral">&quot;/&quot;</span> + input_filename)
<a name="l00130"></a>00130     <span class="keywordflow">return</span> input_file_list
<a name="l00131"></a>00131 
<a name="l00132"></a><a class="code" href="classhadooputil_1_1_hadoop_interface.html#a789db286ea07a0142ba3bfcc5fc94e94">00132</a>   <span class="keyword">def </span><a class="code" href="classhadooputil_1_1_hadoop_interface.html#a789db286ea07a0142ba3bfcc5fc94e94">CatPipe</a>(self, hdfsfiles, pipecmd):
<a name="l00133"></a>00133     catcmd = self.<a class="code" href="classhadooputil_1_1_hadoop_interface.html#af9dab589bb0d3352bf79c662b05bdc7a">hadoopcat_</a> + <span class="stringliteral">&quot; &quot;</span> + hdfsfiles + <span class="stringliteral">&quot; | &quot;</span> + pipecmd
<a name="l00134"></a>00134     if(pyutil.runCommand(catcmd)):
<a name="l00135"></a>00135       pyutil.printError(34, <span class="stringliteral">&quot;Error: &quot;</span> + catcmd)
<a name="l00136"></a>00136 
<a name="l00137"></a><a class="code" href="classhadooputil_1_1_hadoop_interface.html#a6699809315c1001f8e368c25b602f188">00137</a>   <span class="keyword">def </span><a class="code" href="classhadooputil_1_1_hadoop_interface.html#a6699809315c1001f8e368c25b602f188">CatPipeRead</a>(self, hdfsfiles, pipecmd, retval):
<a name="l00138"></a>00138     catcmd = self.<a class="code" href="classhadooputil_1_1_hadoop_interface.html#af9dab589bb0d3352bf79c662b05bdc7a">hadoopcat_</a> + <span class="stringliteral">&quot; &quot;</span> + hdfsfiles + <span class="stringliteral">&quot; | &quot;</span> + pipecmd
<a name="l00139"></a>00139     <span class="keywordflow">if</span> (<span class="keywordflow">not</span> pyutil.readCommand(catcmd, retval)):
<a name="l00140"></a>00140       pyutil.printError(30, <span class="stringliteral">&quot;Error running command &quot;</span> + catcmd)
<a name="l00141"></a>00141 
<a name="l00142"></a>00142   <span class="comment">## RunMR</span>
<a name="l00143"></a>00143   <span class="comment">#  Run a MapReduce</span>
<a name="l00144"></a>00144   <span class="comment">#  @param[in] input_files HDFS location of input files.</span>
<a name="l00145"></a>00145   <span class="comment">#  @param[in] outputdir HDFS location of output.</span>
<a name="l00146"></a>00146   <span class="comment">#  @param[in] reduce_tasks Number of reducer tasks (0 = use default).</span>
<a name="l00147"></a>00147   <span class="comment">#  @param[in] reducer Full string of streaming reducer command.</span>
<a name="l00148"></a>00148   <span class="comment">#  @param[in] mapper Full string of streaming mapper command.</span>
<a name="l00149"></a>00149   <span class="comment">#  @param[in] mroptions Addition streaming MR options (usually specified with -D).</span>
<a name="l00150"></a><a class="code" href="classhadooputil_1_1_hadoop_interface.html#ac71489c42feb74d71f281f2327148608">00150</a>   <span class="keyword">def </span><a class="code" href="classhadooputil_1_1_hadoop_interface.html#ac71489c42feb74d71f281f2327148608" title="RunMR Run a MapReduce.">RunMR</a>(self, input_files, outputdir, reduce_tasks, reducer, mapper, mroptions):
<a name="l00151"></a>00151     mr_str = self.<a class="code" href="classhadooputil_1_1_hadoop_interface.html#a1cf9db2b607bd7159d0abd20e16f98e8">hadoopmr_</a>
<a name="l00152"></a>00152     <span class="keywordflow">if</span> (mroptions):
<a name="l00153"></a>00153       mr_str += mroptions + <span class="stringliteral">&quot; &quot;</span>
<a name="l00154"></a>00154     mr_str += self.<a class="code" href="classhadooputil_1_1_hadoop_interface.html#a89be5c343162d881a0fea7983dedc181">hadooplibpath_</a> + input_files
<a name="l00155"></a>00155     <span class="keywordflow">if</span> <span class="keywordflow">not</span> reducer:
<a name="l00156"></a>00156       mr_str += <span class="stringliteral">&quot; -numReduceTasks 0 --reducer None &quot;</span>
<a name="l00157"></a>00157       <span class="comment">#mr_str += &quot; -numReduceTasks 100 --reducer cat &quot;</span>
<a name="l00158"></a>00158     <span class="keywordflow">else</span>:
<a name="l00159"></a>00159       <span class="keywordflow">if</span> (int(reduce_tasks) &gt;= 0):
<a name="l00160"></a>00160         mr_str += <span class="stringliteral">&quot; -numReduceTasks &quot;</span> + str(reduce_tasks)
<a name="l00161"></a>00161       mr_str += <span class="stringliteral">&quot; --reducer &quot;</span> + reducer
<a name="l00162"></a>00162     mr_str += <span class="stringliteral">&quot; --output &quot;</span> + outputdir
<a name="l00163"></a>00163     mr_str += <span class="stringliteral">&quot; --mapper &quot;</span> + mapper
<a name="l00164"></a>00164     pyutil.printInfo(<span class="stringliteral">&quot;Running MR on: &quot;</span> + input_files)
<a name="l00165"></a>00165     <span class="keywordflow">if</span> (pyutil.runCommand(mr_str) != 0):
<a name="l00166"></a>00166       pyutil.printError(33, <span class="stringliteral">&quot;Error running MR&quot;</span> + mr_str)
</pre></div></div><!-- contents -->
<!-- window showing the filter options -->
<div id="MSearchSelectWindow"
     onmouseover="return searchBox.OnSearchSelectShow()"
     onmouseout="return searchBox.OnSearchSelectHide()"
     onkeydown="return searchBox.OnSearchSelectKey(event)">
<a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(0)"><span class="SelectionMark">&#160;</span>All</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(1)"><span class="SelectionMark">&#160;</span>Classes</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(2)"><span class="SelectionMark">&#160;</span>Namespaces</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(3)"><span class="SelectionMark">&#160;</span>Files</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(4)"><span class="SelectionMark">&#160;</span>Functions</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(5)"><span class="SelectionMark">&#160;</span>Variables</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(6)"><span class="SelectionMark">&#160;</span>Typedefs</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(7)"><span class="SelectionMark">&#160;</span>Enumerations</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(8)"><span class="SelectionMark">&#160;</span>Enumerator</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(9)"><span class="SelectionMark">&#160;</span>Friends</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(10)"><span class="SelectionMark">&#160;</span>Defines</a></div>

<!-- iframe showing the search results (closed by default) -->
<div id="MSearchResultsWindow">
<iframe src="javascript:void(0)" frameborder="0" 
        name="MSearchResults" id="MSearchResults">
</iframe>
</div>



<hr class="footer"/><address class="footer"><small>
Generated on Tue Apr 9 2013 11:56:29 for Reranker Framework (ReFr) by &#160;<a href="http://www.doxygen.org/index.html">
<img class="footer" src="doxygen.png" alt="doxygen"/>
</a> 1.7.6.1
</small></address>

</body>
</html>
